﻿using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
using System.IO;
using System.Net;
using System.Text.RegularExpressions;
using System.Threading;
using System.Collections;
using System.Web;


namespace improveNetDataCollect
{
    public class Crawler
    {
        public Hashtable urls = new Hashtable();
        public int count = 0;
        public string CrawlResult = "";

        public void crawl()
        {
            CrawlResult += "开始爬行了……\n";
            while (true)
            {
                string current = null;
                foreach (string url in urls.Keys)
                {
                    if ((bool)urls[url])
                        continue;
                    current = url;
                }
                if (current == null || count > 10)
                    break;
                CrawlResult += "正爬取" + current + "页面！" + "\n";//显示已爬取url信息
                string html = DownLoad(current);
                urls[current] = true;
                count++;
                if (count == 1)//只爬取起始网站上网页
                    parse(current, html);
            }
            CrawlResult += "爬行结束。";
        }
        public string DownLoad(string url)
        {
            try
            {
                WebClient webclient = new WebClient();
                webclient.Encoding = Encoding.UTF8;
                string html = webclient.DownloadString(url);
                string fileName = count.ToString();
                File.WriteAllText(fileName, html, Encoding.UTF8);
                return html;
            }
            catch (Exception ex)
            {
                CrawlResult += "错误：" + ex.Message + "\n";
                return "";//显示错误的url信息
            }
        }
        public string getAbsoluteUrl(string strRef, string domain)
        {
            if (strRef.Contains("://"))
                return strRef;
            else if (strRef.StartsWith("//"))
                return "https:" + strRef;
            else
            {
                return strRef = "https://" + domain + "/" + strRef;
            }//相对地址转绝对地址
        }
        public void parse(string url, string html)
        {
            string htmlUrlDetect = @"(href|HREF)[]*=[]*[""'][^""'#>]+[""']";//html文本中检测url
            string UrlParse = @"(http|https)://(?<domain>[^(:|/]*)";//解析url正则
            MatchCollection matches = new Regex(htmlUrlDetect).Matches(html);
            Regex regex = new Regex(UrlParse);
            Match m = regex.Match(url);
            string domain = m.Groups["domain"].Value;
            foreach (Match match in matches)
            {
                htmlUrlDetect = match.Value.Substring(match.Value.IndexOf('=') + 1).Trim('"', '\'', '#', ' ', '>');
                if (htmlUrlDetect.Length == 0)
                    continue;
                htmlUrlDetect = getAbsoluteUrl(htmlUrlDetect, domain);
                if (urls[htmlUrlDetect] == null)
                    urls[htmlUrlDetect] = false;

            }
        }
    }
}


